@@ -19,7 +19,7 @@ module Agents |
||
19 | 19 |
"expected_receive_period_in_days": 2, |
20 | 20 |
"trigger_on": "event", |
21 | 21 |
"hit": { |
22 |
- "max_assignments": 1, |
|
22 |
+ "assignments": 1, |
|
23 | 23 |
"title": "Sentiment evaluation", |
24 | 24 |
"description": "Please rate the sentiment of this message: '<$.message>'", |
25 | 25 |
"reward": 0.05, |
@@ -58,7 +58,7 @@ module Agents |
||
58 | 58 |
`default`, `min_length`, and `max_length`. |
59 | 59 |
|
60 | 60 |
If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to |
61 |
- automatically select the majority vote for each question across all `max_assignments`. |
|
61 |
+ automatically select the majority vote for each question across all `assignments`. If all selections are numeric, an `average_answer` will also be generated. |
|
62 | 62 |
|
63 | 63 |
As with most Agents, `expected_receive_period_in_days` is required if `trigger_on` is set to `event`. |
64 | 64 |
MD |
@@ -71,7 +71,14 @@ module Agents |
||
71 | 71 |
MD |
72 | 72 |
|
73 | 73 |
def validate_options |
74 |
+ options[:hit] ||= {} |
|
75 |
+ options[:hit][:questions] ||= [] |
|
76 |
+ |
|
74 | 77 |
errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on]) |
78 |
+ errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options[:hit][:assignments].present? && options[:hit][:assignments].to_i > 0 |
|
79 |
+ errors.add(:base, "'hit.title' must be provided") unless options[:hit][:title].present? |
|
80 |
+ errors.add(:base, "'hit.description' must be provided") unless options[:hit][:description].present? |
|
81 |
+ errors.add(:base, "'hit.questions' must be provided") unless options[:hit][:questions].present? && options[:hit][:questions].length > 0 |
|
75 | 82 |
|
76 | 83 |
if options[:trigger_on] == "event" |
77 | 84 |
errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present? |
@@ -79,6 +86,14 @@ module Agents |
||
79 | 86 |
errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0 |
80 | 87 |
end |
81 | 88 |
|
89 |
+ if options[:hit][:questions].any? { |question| [:key, :name, :required, :type, :question].any? {|k| !question[k].present? } } |
|
90 |
+ errors.add(:base, "all questions must set 'key', 'name', 'required', 'type', and 'question'") |
|
91 |
+ end |
|
92 |
+ |
|
93 |
+ if options[:hit][:questions].any? { |question| question[:type] == "selection" && (!question[:selections].present? || question[:selections].length == 0 || !question[:selections].all? {|s| s[:key].present? } || !question[:selections].all? { |s| s[:text].present? })} |
|
94 |
+ errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'") |
|
95 |
+ end |
|
96 |
+ |
|
82 | 97 |
if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" } |
83 | 98 |
errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option") |
84 | 99 |
end |
@@ -90,7 +105,7 @@ module Agents |
||
90 | 105 |
:trigger_on => "event", |
91 | 106 |
:hit => |
92 | 107 |
{ |
93 |
- :max_assignments => 1, |
|
108 |
+ :assignments => 1, |
|
94 | 109 |
:title => "Sentiment evaluation", |
95 | 110 |
:description => "Please rate the sentiment of this message: '<$.message>'", |
96 | 111 |
:reward => 0.05, |
@@ -225,7 +240,7 @@ module Agents |
||
225 | 240 |
description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip |
226 | 241 |
questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload) |
227 | 242 |
hit = RTurk::Hit.create(:title => title) do |hit| |
228 |
- hit.max_assignments = (options[:hit][:max_assignments] || 1).to_i |
|
243 |
+ hit.max_assignments = (options[:hit][:assignments] || 1).to_i |
|
229 | 244 |
hit.description = description |
230 | 245 |
hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions) |
231 | 246 |
hit.reward = (options[:hit][:reward] || 0.05).to_f |
@@ -17,6 +17,97 @@ describe Agents::HumanTaskAgent do |
||
17 | 17 |
end |
18 | 18 |
|
19 | 19 |
describe "validations" do |
20 |
+ it "validates that trigger_on is 'schedule' or 'event'" do |
|
21 |
+ @checker.options[:trigger_on] = "foo" |
|
22 |
+ @checker.should_not be_valid |
|
23 |
+ end |
|
24 |
+ |
|
25 |
+ it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do |
|
26 |
+ @checker.options[:trigger_on] = "event" |
|
27 |
+ @checker.options[:expected_receive_period_in_days] = nil |
|
28 |
+ @checker.should_not be_valid |
|
29 |
+ @checker.options[:expected_receive_period_in_days] = 2 |
|
30 |
+ @checker.should be_valid |
|
31 |
+ end |
|
32 |
+ |
|
33 |
+ it "requires a positive submission_period when trigger_on is set to 'schedule'" do |
|
34 |
+ @checker.options[:trigger_on] = "schedule" |
|
35 |
+ @checker.options[:submission_period] = nil |
|
36 |
+ @checker.should_not be_valid |
|
37 |
+ @checker.options[:submission_period] = 2 |
|
38 |
+ @checker.should be_valid |
|
39 |
+ end |
|
40 |
+ |
|
41 |
+ it "requires a hit.title" do |
|
42 |
+ @checker.options[:hit][:title] = "" |
|
43 |
+ @checker.should_not be_valid |
|
44 |
+ end |
|
45 |
+ |
|
46 |
+ it "requires a hit.description" do |
|
47 |
+ @checker.options[:hit][:description] = "" |
|
48 |
+ @checker.should_not be_valid |
|
49 |
+ end |
|
50 |
+ |
|
51 |
+ it "requires hit.assignments" do |
|
52 |
+ @checker.options[:hit][:assignments] = "" |
|
53 |
+ @checker.should_not be_valid |
|
54 |
+ @checker.options[:hit][:assignments] = 0 |
|
55 |
+ @checker.should_not be_valid |
|
56 |
+ @checker.options[:hit][:assignments] = "moose" |
|
57 |
+ @checker.should_not be_valid |
|
58 |
+ @checker.options[:hit][:assignments] = "2" |
|
59 |
+ @checker.should be_valid |
|
60 |
+ end |
|
61 |
+ |
|
62 |
+ it "requires hit.questions" do |
|
63 |
+ old_questions = @checker.options[:hit][:questions] |
|
64 |
+ @checker.options[:hit][:questions] = nil |
|
65 |
+ @checker.should_not be_valid |
|
66 |
+ @checker.options[:hit][:questions] = [] |
|
67 |
+ @checker.should_not be_valid |
|
68 |
+ @checker.options[:hit][:questions] = [old_questions[0]] |
|
69 |
+ @checker.should be_valid |
|
70 |
+ end |
|
71 |
+ |
|
72 |
+ it "requires that all questions have key, name, required, type, and question" do |
|
73 |
+ old_questions = @checker.options[:hit][:questions] |
|
74 |
+ @checker.options[:hit][:questions].first[:key] = "" |
|
75 |
+ @checker.should_not be_valid |
|
76 |
+ |
|
77 |
+ @checker.options[:hit][:questions] = old_questions |
|
78 |
+ @checker.options[:hit][:questions].first[:name] = "" |
|
79 |
+ @checker.should_not be_valid |
|
80 |
+ |
|
81 |
+ @checker.options[:hit][:questions] = old_questions |
|
82 |
+ @checker.options[:hit][:questions].first[:required] = nil |
|
83 |
+ @checker.should_not be_valid |
|
84 |
+ |
|
85 |
+ @checker.options[:hit][:questions] = old_questions |
|
86 |
+ @checker.options[:hit][:questions].first[:type] = "" |
|
87 |
+ @checker.should_not be_valid |
|
88 |
+ |
|
89 |
+ @checker.options[:hit][:questions] = old_questions |
|
90 |
+ @checker.options[:hit][:questions].first[:question] = "" |
|
91 |
+ @checker.should_not be_valid |
|
92 |
+ end |
|
93 |
+ |
|
94 |
+ it "requires that all questions of type 'selection' have a selections array with keys and text" do |
|
95 |
+ @checker.options[:hit][:questions][0][:selections] = [] |
|
96 |
+ @checker.should_not be_valid |
|
97 |
+ @checker.options[:hit][:questions][0][:selections] = [{}] |
|
98 |
+ @checker.should_not be_valid |
|
99 |
+ @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "" }] |
|
100 |
+ @checker.should_not be_valid |
|
101 |
+ @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "hi" }] |
|
102 |
+ @checker.should_not be_valid |
|
103 |
+ @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "" }] |
|
104 |
+ @checker.should_not be_valid |
|
105 |
+ @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }] |
|
106 |
+ @checker.should be_valid |
|
107 |
+ @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }, {}] |
|
108 |
+ @checker.should_not be_valid |
|
109 |
+ end |
|
110 |
+ |
|
20 | 111 |
it "requires that all questions be of type 'selection' when `take_majority` is `true`" do |
21 | 112 |
@checker.options[:take_majority] = "true" |
22 | 113 |
@checker.should_not be_valid |
@@ -91,7 +182,7 @@ describe Agents::HumanTaskAgent do |
||
91 | 182 |
|
92 | 183 |
@checker.send :create_hit, @event |
93 | 184 |
|
94 |
- hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments] |
|
185 |
+ hitInterface.max_assignments.should == @checker.options[:hit][:assignments] |
|
95 | 186 |
hitInterface.reward.should == @checker.options[:hit][:reward] |
96 | 187 |
hitInterface.description.should == "Make something for Joe" |
97 | 188 |
|
@@ -110,7 +201,7 @@ describe Agents::HumanTaskAgent do |
||
110 | 201 |
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) |
111 | 202 |
mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface } |
112 | 203 |
@checker.send :create_hit |
113 |
- hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments] |
|
204 |
+ hitInterface.max_assignments.should == @checker.options[:hit][:assignments] |
|
114 | 205 |
hitInterface.reward.should == @checker.options[:hit][:reward] |
115 | 206 |
end |
116 | 207 |
end |